#!/bin/sh

KIND_NAME="${KIND_NAME:-kind}"
KIND_NODES="${KIND_NODES:-1}"
KIND_LOCK_PATH="${KIND_LOCK_PATH:-$TARGET_PATH/kind-lock}"
KIND_CONTAINERD_CACHE_PATH="${K8S_CACHE_PATH:-$KIND_CONTAINERD_CACHE_PATH}"
KIND_CONTAINERD_CACHE_RESET="${K8S_CACHE_RESET:-$KIND_CONTAINERD_CACHE_RESET}"
KIND_LOG_PATH="${KIND_LOG_PATH:-$TARGET_PATH/kind-logs}"
KIND_LOG_RESOURCES_POLICY_PATH="${KIND_LOG_RESOURCES_POLICY_PATH:-$TARGET_PATH/kind-apiserver-audit-policy}"
KIND_EXPANDABLE_STORAGE_CLASSNAME="${EXPANDABLE_STORAGE_CLASSNAME:-expandable-sc}"
K8S_IP_FAMILY="${K8S_IP_FAMILY:-ipv4}"
KIND_0_15_0="${KIND_0_15_0:-kind-0.15.0}"
KIND_0_20_0="${KIND_0_20_0:-kind-0.20.0}"
KIND_0_23_0="${KIND_0_23_0:-kind-0.23.0}"
KIND_0_25_0="${KIND_0_25_0:-kind-0.25.0}"
KIND_0_26_0="${KIND_0_26_0:-kind-0.26.0}"
KIND_0_29_0="${KIND_0_29_0:-kind-0.29.0}"
KIND_0_30_0="${KIND_0_30_0:-kind-0.30.0}"

check_kind_version() {
  if [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.31" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    KIND="$KIND_0_30_0"
    if ! "$KIND" version | grep -q -F 'kind v0.30.0 '
    then
      echo "To run Kubernetes 1.31+ kind v0.30.0 is required"
      return 1
    fi
  elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.30" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    KIND="$KIND_0_29_0"
    if ! "$KIND" version | grep -q -F 'kind v0.29.0 '
    then
      echo "To run Kubernetes 1.30+ kind v0.29.0 is required"
      return 1
    fi
  elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.29" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    KIND="$KIND_0_26_0"
    if ! "$KIND" version | grep -q -F 'kind v0.26.0 '
    then
      echo "To run Kubernetes 1.29+ kind v0.26.0 is required"
      return 1
    fi
  elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.26" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    KIND="$KIND_0_25_0"
    if ! "$KIND" version | grep -q -F 'kind v0.25.0 '
    then
      echo "To run Kubernetes 1.26+ kind v0.25.0 is required"
      return 1
    fi
  elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.25" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    KIND="$KIND_0_23_0"
    if ! "$KIND" version | grep -q -F 'kind v0.23.0 '
    then
      echo "To run Kubernetes 1.25+ kind v0.23.0 is required"
      return 1
    fi
  elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.21" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    KIND="$KIND_0_20_0"
    if ! "$KIND" version | grep -q -F 'kind v0.20.0 '
    then
      echo "To run Kubernetes 1.21+ kind v0.20.0 is required"
      return 1
    fi
  else
    KIND="$KIND_0_15_0"
    if ! "$KIND" version | grep -q -F 'kind v0.15.0 '
    then
      echo "To run Kubernetes 1.20- kind v0.15.0 is required"
      return 1
    fi
  fi
}

get_k8s_env_version() {
  echo "Kind version $("$KIND" version | cut -d ' ' -f 2)"
  echo
}

update_k8s_config() {
  check_kind_version

  mkdir -p "$HOME/.kube"
  if [ "$K8S_FROM_DIND" = true ]
  then
    if docker network ls --format '{{ .Name }}' | grep -q '^kind$'
    then
      local CONTAINER_NAME
      CONTAINER_NAME="$(cat /proc/self/cgroup | grep '^1:name' | cut -d / -f 3)"
      CONTAINER_NAME="${CONTAINER_NAME:-$(docker ps --filter name="^$(cat /etc/hostname)" --format '{{.Names}}')}"
      CONTAINER_NAME="${CONTAINER_NAME:-$(docker ps --filter id="$(cat /etc/hostname)" --format '{{.ID}}')}"
      docker inspect "$CONTAINER_NAME" \
        -f '{{ range $key,$value := .NetworkSettings.Networks }}{{ printf "%s\n" $key }}{{ end }}' \
        | grep -q '^kind$' \
        || docker network connect kind "$CONTAINER_NAME"
    fi
    local KIND_CONTROL_PLANE_IP
    KIND_CONTROL_PLANE_IP="$(docker inspect "$KIND_NAME-control-plane" \
      -f '{{ .NetworkSettings.Networks.kind.IPAddress }}')"
    "$KIND" get kubeconfig --name "$KIND_NAME" --internal \
      | sed "s/$KIND_NAME-control-plane/$KIND_CONTROL_PLANE_IP/" \
      > "$HOME/.kube/config-$KIND_NAME"
  else
    "$KIND" get kubeconfig --name "$KIND_NAME" \
      > "$HOME/.kube/config-$KIND_NAME"
  fi

  (
  export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
  if [ -s "$KUBECONFIG" ]
  then
    KUBECONFIG="$HOME/.kube/config-$KIND_NAME":"$KUBECONFIG" \
      kubectl config view --raw > "$HOME/.kube/config-merged"
    mv "$HOME/.kube/config-merged" "$KUBECONFIG"
  else
    mv "$HOME/.kube/config-$KIND_NAME" "$KUBECONFIG"
  fi
  chmod 700 "$KUBECONFIG"
  )

  chmod 700 "$KUBECONFIG"
  # fix for Unable to connect to the server: x509: certificate is valid for <ips>, not <ip>
  kubectl config set "clusters.kind-$KIND_NAME.insecure-skip-tls-verify" --set-raw-bytes true
  kubectl config unset "clusters.kind-$KIND_NAME.certificate-authority-data"

  screen -ls 2>/dev/null | grep '\.kind-tag-import-images' \
    | tr '[:space:]' ' ' | cut -d ' ' -f 2 | cut -d . -f 1 \
    | xargs -r -I % kill % || true
  E2E_ENV=kind screen -dmS kind-tag-import-images -L -Logfile "$HOME/.kind-tag-import-images.log" "$SHELL" stackgres-k8s/e2e/e2e properly_tag_not_found_import_images
  if [ "$K8S_IP_FAMILY" = ipv6 ]
  then
    screen -ls 2>/dev/null | grep '\.kind-ipv6-dns-socat' \
      | tr '[:space:]' ' ' | cut -d ' ' -f 2 | cut -d . -f 1 \
      | xargs -r -I % kill %
    screen -dmS kind-ipv6-dns-socat -L -Logfile "$HOME/.crc/kind-ipv6-dns-socat.log" socat UDP6-RECVFROM:5300,fork UDP4-SENDTO:127.0.0.53:53
  fi
}

properly_tag_not_found_import_images() {
  echo "Looking for import-* images to tag properly"
  event_watch  --follow \
    | stdbuf -o0 grep '\simage "\(.*library/import-[^@]\+@sha256:[^"]\+\)": not found' \
    | stdbuf -o0 sed 's#^.*\simage "\(.*library/import-[^@]\+@sha256:[^"]\+\)": not found.*$#\1#' \
    | (
      while read IMAGE_NAME
      do
        echo "Detected import-* image $IMAGE_NAME to tag properly into k8s env $E2E_ENV"
        tag_image_k8s "${IMAGE_NAME#*library/}" "$IMAGE_NAME"
      done
      )
}

reuse_k8s() {
  check_kind_version
  check_kind_image_exists

  try_function update_k8s_config

  if ! "$KIND" get clusters | grep -q "^$KIND_NAME$" \
      || ! docker inspect "$KIND_NAME-control-plane" -f '{{ .State.Status }}' \
        | grep -q -F 'running' \
      || ! docker inspect "$KIND_NAME-control-plane" -f '{{ .Config.Image }}' \
        | grep -q -F "kindest/node:v$(get_kind_image "${K8S_VERSION}")"
  then
    echo "Can not reuse kind environment $KIND_NAME"
    reset_k8s
    return
  fi

  if [ -n "$E2E_EXTRA_MOUNT_BUILD_PATH" ]
  then
    k8s_copy_to_extra_mount_build_path
  fi

  echo "Reusing kind environment $KIND_NAME"
}

reset_k8s() {
  check_kind_version
  check_kind_image_exists

  if [ -n "$E2E_EXTRA_MOUNT_BUILD_PATH" ]
  then
    k8s_copy_to_extra_mount_build_path
    KIND_EXTRA_MOUNTS="$KIND_EXTRA_MOUNTS
      $(realpath "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/operator/target/quarkus-app)
      $(realpath "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/restapi/target/quarkus-app)
      $(realpath "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/admin-ui/target/public)
      $(realpath "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/cluster-controller/target/quarkus-app)
      $(realpath "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/stream/target/quarkus-app)"
  fi

  echo "Setting up kind environment $KIND_NAME..."

  if [ -n "$KIND_CONTAINERD_CACHE_PATH" ]
  then
    echo "Setting up kind containerd cache in $KIND_CONTAINERD_CACHE_PATH..."
    if [ "$KIND_CONTAINERD_CACHE_RESET" = true ]
    then
      docker run --rm -v "$KIND_CONTAINERD_CACHE_PATH:/containerd-cache" alpine \
        sh -c 'rm -rf /containerd-cache/*'
    fi
  fi

  if [ -n "$K8S_EXTRA_PORT" ]
  then
    echo "Setting up kind port $K8S_EXTRA_PORT..."
  fi

  delete_k8s
  if [ "$KIND_LOG_RESOURCES" = true ]
  then
    cat << EOF > "$KIND_LOG_RESOURCES_POLICY_PATH"
# Log all requests at the RequestResponse level.
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: RequestResponse
EOF
  fi
  cat << EOF > "$TARGET_PATH/kind-config.yaml"
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
featureGates:
$(
  if [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.20" | tr . '\n' | xargs -I @ printf '%05d' @)" ] \
      && [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -lt "$(echo "1.32" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    cat << INNER_INNER_EOF
  HPAContainerMetrics: true
INNER_INNER_EOF
  fi
)
$(
  if [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -ge "$(echo "1.27" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    cat << INNER_INNER_EOF
  InPlacePodVerticalScaling: true
INNER_INNER_EOF
  fi
)
$(
  if [ -n "$KIND_CONTAINERD_CACHE_PATH" ] || \
    docker system info 2> /dev/null | grep -q "Backing Filesystem: \(zfs\|btrfs\)"
  then
    cat << INNER_EOF
containerdConfigPatches:
- |-
$(
  if [ -n "$KIND_CONTAINERD_CACHE_PATH" ]
  then
    cat << INNER_INNER_EOF
 
 root = "/containerd-cache"
INNER_INNER_EOF
  fi
  if docker system info 2> /dev/null | grep -q "Backing Filesystem: zfs" \
    || ([ -d "$KIND_CONTAINERD_CACHE_PATH" ] \
      && df -T "$KIND_CONTAINERD_CACHE_PATH" | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2 | grep -q '^zfs$')
  then
    cat << INNER_INNER_EOF
 [plugins."io.containerd.grpc.v1.cri".containerd]
 snapshotter = "zfs"
INNER_INNER_EOF
  fi
  if docker system info 2> /dev/null | grep -q "Backing Filesystem: btrfs" \
    || ([ -d "$KIND_CONTAINERD_CACHE_PATH" ] \
      && df -T "$KIND_CONTAINERD_CACHE_PATH" | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2 | grep -q '^btrfs$')
  then
    cat << INNER_INNER_EOF
 [plugins."io.containerd.grpc.v1.cri".containerd]
 snapshotter = "btrfs"
INNER_INNER_EOF
  fi
)
INNER_EOF
  fi
)
networking:
  ipFamily: "${K8S_IP_FAMILY}"
  disableDefaultCNI: $(if [ "$K8S_DISABLE_CALICO" != true ]; then printf true; else printf false; fi)
  apiServerAddress: "0.0.0.0"
nodes:
- role: control-plane
  labels:
    ingress-ready: true
  kubeadmConfigPatches:
  - |
    kind: ClusterConfiguration
    apiServer:
        extraArgs:
          enable-admission-plugins: NodeRestriction,OwnerReferencesPermissionEnforcement
$(
  if [ "$KIND_LOG_RESOURCES" = true ]
  then
    cat << INNER_EOF
          # enable auditing flags on the API server
          audit-log-path: /var/log/kubernetes/kube-apiserver-audit.log
          audit-policy-file: /etc/kubernetes/policies/audit-policy.yaml
        # mount new files / directories on the control plane
        extraVolumes:
          - name: audit-policies
            hostPath: /etc/kubernetes/policies
            mountPath: /etc/kubernetes/policies
            readOnly: true
            pathType: "DirectoryOrCreate"
          - name: "audit-logs"
            hostPath: "/var/log/kubernetes"
            mountPath: "/var/log/kubernetes"
            readOnly: false
            pathType: DirectoryOrCreate
INNER_EOF
  fi
  if [ -n "$K8S_EXTRA_PORT" ]
  then
    cat << INNER_EOF
  extraPortMappings:
  - containerPort: $(echo "$K8S_EXTRA_PORT" | cut -d : -f 1)
    hostPort: $(echo "$K8S_EXTRA_PORT" | cut -d : -f 2)
    listenAddress: "$(echo "$K8S_EXTRA_PORT" | cut -d : -f 3)"
    protocol: "$(echo "$K8S_EXTRA_PORT" | cut -d : -f 4)"
INNER_EOF
  fi
  if [ -n "$KIND_EXTRA_MOUNTS" ] \
    || [ -n "$KIND_CONTAINERD_CACHE_PATH" ] \
    || [ "$KIND_LOG" = true ] \
    || [ "$KIND_LOG_RESOURCES" = true ]
  then
    cat << INNER_EOF
  extraMounts:
INNER_EOF
  fi
  if [ -n "$KIND_EXTRA_MOUNTS" ]
  then
    for KIND_EXTRA_MOUNT in $KIND_EXTRA_MOUNTS
    do
      cat << INNER_EOF
  - hostPath: $KIND_EXTRA_MOUNT
    containerPath: $KIND_EXTRA_MOUNT
INNER_EOF
    done
  fi
  if [ -n "$KIND_CONTAINERD_CACHE_PATH" ]
  then
    mkdir -p "$KIND_CONTAINERD_CACHE_PATH"
    cat << INNER_EOF
  - hostPath: $(realpath "$KIND_CONTAINERD_CACHE_PATH")
    containerPath: /containerd-cache
INNER_EOF
  fi
  if [ "$KIND_LOG" = true ]
  then
    mkdir -p "$KIND_LOG_PATH"
    cat << INNER_EOF
  - hostPath: $KIND_LOG_PATH
    containerPath: /var/log
INNER_EOF
  fi
  if [ "$KIND_LOG_RESOURCES" = true ]
  then
    cat << INNER_EOF
  - hostPath: $KIND_LOG_RESOURCES_POLICY_PATH
    containerPath: /etc/kubernetes/policies/audit-policy.yaml
    readOnly: true
INNER_EOF
  fi
  for KIND_NODE in $(seq 2 "$KIND_NODES")
  do
    cat << INNER_EOF
- role: worker
INNER_EOF
  done
)
EOF

  try_function flock "$KIND_LOCK_PATH" \
    "$KIND" create cluster --name "$KIND_NAME" --config "$TARGET_PATH/kind-config.yaml" \
    --image "kindest/node:v$(get_kind_image "${K8S_VERSION}")"
  if ! "$RESULT" && [ -n "$KIND_CONTAINERD_CACHE_PATH" ] && [ "$KIND_CONTAINERD_CACHE_RESET" != true ]
  then
    echo "Kind failed to create cluster with cache enabled. Resetting cache and retrying!"
    KIND_CONTAINERD_CACHE_RESET=true reset_k8s
    return
  fi

  if [ "$KIND_INSTALL_NFS" = "true" ]
  then
    echo "Setting up NFT tools for kind..."
    "$KIND" get nodes --name "$KIND_NAME" \
      | xargs -r -n 1 -I % -P "$E2E_PARALLELISM" sh -ec "
      docker exec '%' sh -c 'DEBIAN_FRONTEND=noninteractive apt-get update -y -qq < /dev/null > /dev/null'
      docker exec '%' sh -c 'DEBIAN_FRONTEND=noninteractive apt-get install -y -qq nfs-common < /dev/null > /dev/null'
      "
  fi

  update_k8s_config

  if [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
      -eq "$(echo "1.12" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
  then
    echo "Patch coredns to version 1.3.1 (see https://github.com/coredns/coredns/issues/2391)..."
    kubectl patch deployment -n kube-system coredns --type json \
      --patch '[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"k8s.gcr.io/coredns:1.3.1"}]'
  fi
  if [ "$K8S_IP_FAMILY" = ipv6 ]
  then
    DOCKER_IPV6="$(docker inspect "$KIND_NAME-control-plane" --format='{{range .NetworkSettings.Networks}}{{.IPv6Gateway}}{{end}}')"
    kubectl get -oyaml -n=kube-system configmap/coredns \
      | sed "s/forward .*$/forward . [$DOCKER_IPV6]:5300 {/" \
      | kubectl replace -f -
  fi

  if [ "$K8S_DISABLE_CALICO" != true ]
  then
    echo "Setting up calico for kind..."
    until kubectl get node --template '{{ if (index .items 0).spec.podCIDR }}true{{ end }}' | grep -q 'true'
    do
      sleep 3
    done
    K8S_POD_CIDR="$(kubectl get node --template '{{ (index .items 0).spec.podCIDR }}')"
    if [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.22" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      kubectl replace --force -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/tigera-operator.yaml
      wait_until kubectl  replace --force -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.4/manifests/custom-resources.yaml
      kubectl patch installations.operator.tigera.io default --type json \
        -p '[{"op":"replace","path":"/spec/calicoNetwork/ipPools/0/cidr","value":"'"$K8S_POD_CIDR"'"}]'
    else
      kubectl apply -f https://docs.projectcalico.org/v3.12/manifests/calico.yaml
      kubectl -n kube-system set env daemonset/calico-node CALICO_IPV4POOL_CIDR="$K8S_POD_CIDR"
      kubectl -n kube-system set env daemonset/calico-node FELIX_IGNORELOOSERPF=true
    fi
    echo "...done"
  fi

  if [ "$K8S_DISABLE_VOLUME_SNAPSHOT" != true ]
  then
    echo "Setting up CSI Hostpath Driver with Snapshotter for kind..."
    CSI_DRIVER_HOST_PATH_VERSION=""
    if [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.29" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=master
      SNAPSHOTTER_VERSION=v6.3.2
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.26" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.12.0
      SNAPSHOTTER_VERSION=v6.3.2
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.24" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.11.0
      SNAPSHOTTER_VERSION=v6.3.2
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.22" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.10.0
      SNAPSHOTTER_VERSION=v6.3.2
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.21" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.9.0
      SNAPSHOTTER_VERSION=v6.3.2
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.20" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.9.0
      SNAPSHOTTER_VERSION=v3.0.3
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.19" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.7.3
      SNAPSHOTTER_VERSION=v3.0.3
    elif [ "$(echo "$K8S_VERSION" | tr . '\n' | head -n 2 | xargs -I @ printf '%05d' @)" \
        -ge "$(echo "1.18" | tr . '\n' | xargs -I @ printf '%05d' @)" ]
    then
      CSI_DRIVER_HOST_PATH_VERSION=v1.7.2
      SNAPSHOTTER_VERSION=v3.0.3
    fi
    if [ -z "$CSI_DRIVER_HOST_PATH_VERSION" ]
    then
      echo "Can not find a valid csi driver host path version"
      exit 1
    fi
    if ! git -C "$TARGET_PATH/csi-driver-host-path/.git" diff origin/"$CSI_DRIVER_HOST_PATH_VERSION" 2>/dev/null
    then
      rm -rf "$TARGET_PATH/csi-driver-host-path"
      git clone -b "$CSI_DRIVER_HOST_PATH_VERSION" --single-branch \
        https://github.com/kubernetes-csi/csi-driver-host-path \
        "$TARGET_PATH/csi-driver-host-path"
    fi
    
    {
      # Apply VolumeSnapshot CRDs
      kubectl replace --force -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml"
      kubectl replace --force -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml"
      kubectl replace --force -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml"
      # Create snapshot controller
      kubectl replace --force -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml"
      kubectl replace --force -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
      CSI_DRIVER_HOST_PATH_PATH="$TARGET_PATH/csi-driver-host-path/deploy/kubernetes-$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)"
      if [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.20 ]
      then
        CSI_DRIVER_HOST_PATH_PATH="$TARGET_PATH/csi-driver-host-path/deploy/kubernetes-1.21"
      elif [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.23 ]
      then
        CSI_DRIVER_HOST_PATH_PATH="$TARGET_PATH/csi-driver-host-path/deploy/kubernetes-1.22"
      elif [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.29 ] \
        || [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.30 ] \
        || [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.31 ] \
        || [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.32 ] \
        || [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.33 ] \
        || [ "$(printf %s "$K8S_VERSION" | cut -d . -f 1-2)" = 1.34 ]
      then
        CSI_DRIVER_HOST_PATH_PATH="$TARGET_PATH/csi-driver-host-path/deploy/kubernetes-1.30"
      fi
      cat << 'EOF' > "$CSI_DRIVER_HOST_PATH_PATH/kubectlw"
set -e
if [ "x$1" = xapply ]
then
  if [ "$#" = 3 ] && [ "x$2" = x--kustomize ]
  then
    for YAML in "$3"/*.yaml
    do
      echo "Mocking $YAML"
      yq -y -c '
          .
            | if (. | type) == "array" then .[] else . end
            | if (.|has("metadata")) then .metadata.namespace = "kube-system" else . end
          ' "$YAML"  \
        | sed 's#namespace: default#namespace: kube-system#' > "$YAML.tmp"
      mv "$YAML.tmp" "$YAML"
    done
    kubectl -n kube-system apply --kustomize "$3"
  elif [ "$#" = 3 ] && [ "x$2 $3" = 'x-f -' ]
  then
      echo "Mocking stdin"
      yq -y -c '
          .
            | if (. | type) == "array" then .[] else . end
            | if (.|has("metadata")) then .metadata.namespace = "kube-system" else . end
          '  \
        | sed 's#namespace: default#namespace: kube-system#' \
        | kubectl -n kube-system apply -f -
  else
    printf '%s\n' "No mock for $*"
    exit 1
  fi
else
  kubectl -n kube-system "$@"
fi
EOF
      sed -i "s#kubectl#sh $CSI_DRIVER_HOST_PATH_PATH/kubectlw#" \
        "$CSI_DRIVER_HOST_PATH_PATH"/deploy.sh
      IMAGE_TAG= bash "$CSI_DRIVER_HOST_PATH_PATH"/deploy.sh
      kubectl replace --force -f "$TARGET_PATH/csi-driver-host-path/examples/csi-storageclass.yaml"
      kubectl get storageclass -o name | xargs -I % kubectl annotate % --overwrite storageclass.kubernetes.io/is-default-class=false
      kubectl annotate storageclass csi-hostpath-sc --overwrite storageclass.kubernetes.io/is-default-class=true
      kubectl annotate volumesnapshotclass csi-hostpath-snapclass --overwrite snapshot.storage.kubernetes.io/is-default-class="true"
      kubectl patch sts -n kube-system csi-hostpathplugin --type json -p '[{"op":"add","path":"/spec/template/spec/containers/7/args/2","value":"--timeout=3m"}]'
    }
    echo "...done"
  fi

  if [ "$ENABLE_METRIC_SERVER" = true ]
  then
    setup_metric_server
  fi
}

k8s_copy_to_extra_mount_build_path() {
  if [ -z "$PROJECT_PATH" ]
  then
    echo "PROJECT_PATH not set"
    return 1
  fi
  if [ -z "$E2E_EXTRA_MOUNT_BUILD_PATH" ]
  then
    echo "E2E_EXTRA_MOUNT_BUILD_PATH not set"
    return 1
  fi
  mkdir -p "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/operator/target/quarkus-app
  cp -a "$PROJECT_PATH"/stackgres-k8s/src/operator/target/quarkus-app/. \
    "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/operator/target/quarkus-app/.
  mkdir -p "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/restapi/target/quarkus-app
  cp -a "$PROJECT_PATH"/stackgres-k8s/src/restapi/target/quarkus-app/. \
    "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/restapi/target/quarkus-app/.
  mkdir -p "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/admin-ui/target/public
  cp -a "$PROJECT_PATH"/stackgres-k8s/src/admin-ui/target/public/. \
    "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/admin-ui/target/public/.
  mkdir -p "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/cluster-controller/target/quarkus-app
  cp -a "$PROJECT_PATH"/stackgres-k8s/src/cluster-controller/target/quarkus-app/. \
    "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/cluster-controller/target/quarkus-app/.
  mkdir -p "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/stream/target/quarkus-app
  cp -a "$PROJECT_PATH"/stackgres-k8s/src/stream/target/quarkus-app/. \
    "$E2E_EXTRA_MOUNT_BUILD_PATH"/stackgres-k8s/src/stream/target/quarkus-app/.
}

setup_metric_server() {
  echo "Setting up metric server for kind..."
  kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
  kubectl patch -n kube-system deploy metrics-server --type json \
    -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/5","value":"--kubelet-insecure-tls"}]'
  echo "...done"
}

delete_k8s() {
  check_kind_version

  if "$KIND" get clusters | grep -q "^$KIND_NAME$"
  then
    echo "Deleting kind environment $KIND_NAME..."

    "$KIND" delete cluster --name "$KIND_NAME" || true

    echo "...done"
  fi

  screen -ls 2>/dev/null | grep '\.kind-tag-import-images' \
    | tr '[:space:]' ' ' | cut -d ' ' -f 2 | cut -d . -f 1 \
    | xargs -r -I % kill % || true
  if [ "$K8S_IP_FAMILY" = ipv6 ]
  then
    screen -ls 2>/dev/null | grep '\.kind-ipv6-dns-socat' \
      | tr '[:space:]' ' ' | cut -d ' ' -f 2 | cut -d . -f 1 \
      | xargs -r -I % kill % || true
  fi
}

has_image_k8s() {
  check_kind_version

  local KIND_IMAGE_ID
  KIND_IMAGE_ID="$( (docker exec "${KIND_NAME}-control-plane" crictl inspecti -o json "$1" 2>/dev/null || printf '{"status": {"id": "unknown"}}') | jq -r '.status.id' | grep -v '^$')"
  if [ "$KIND_IMAGE_ID" != unknown ]
  then
    return
  fi
  return 1
}

load_image_k8s() {
  check_kind_version

  local IMAGE_ID
  IMAGE_ID="$( (docker inspect --format '{{ .ID }}' "$1" 2>/dev/null || printf unknown) | grep -v '^$')"
  local KIND_IMAGE_ID
  KIND_IMAGE_ID="$( (docker exec "${KIND_NAME}-control-plane" crictl inspecti -o json "$1" 2>/dev/null || printf '{"status": {"id": "unknown"}}') | jq -r '.status.id' | grep -v '^$')"
  if [ "$IMAGE_ID" = unknown ] && [ "$KIND_IMAGE_ID" != unknown ]
  then
    echo "Image $1 already loaded in kind environemnt $KIND_NAME"
    return
  fi
  if [ "$KIND_IMAGE_ID" = "$IMAGE_ID" ] && [ "$IMAGE_ID" != unknown ]
  then
    echo "Image $1 already loaded in kind environemnt $KIND_NAME"
    return
  fi
  "$KIND" load docker-image --name "$KIND_NAME" "$1"

  echo "Loaded image $1 in kind environemnt $KIND_NAME"
}

pull_image_k8s() {
  check_kind_version

  local IMAGE_ID
  IMAGE_ID="$( (docker inspect --format '{{ .ID }}' "$1" 2>/dev/null || printf unknown) | grep -v '^$')"
  local KIND_IMAGE_ID
  KIND_IMAGE_ID="$( (docker exec "${KIND_NAME}-control-plane" crictl inspecti -o json "$1" 2>/dev/null || printf '{"status": {"id": "unknown"}}') | jq -r '.status.id' | grep -v '^$')"
  if [ "$IMAGE_ID" = unknown ] && [ "$KIND_IMAGE_ID" != unknown ]
  then
    echo "Image $1 already loaded in kind environemnt $KIND_NAME"
    return
  fi
  if [ "$KIND_IMAGE_ID" = "$IMAGE_ID" ] && [ "$IMAGE_ID" != unknown ]
  then
    echo "Image $1 already loaded in kind environemnt $KIND_NAME"
    return
  fi

  local AUTH
  AUTH="$(jq -r '.auths|to_entries|.[]|.key + "|" + .value.auth' "${HOME}/.docker/config.json" \
    | grep -F "${1%%/*}" | head -n 1 | cut -d '|' -f 2)"
  if [ -n "$AUTH" ]
  then
    docker exec "${KIND_NAME}-control-plane" ctr -n k8s.io i pull --user "$(printf %s "$AUTH" | base64 -d)" "$1" > /dev/null
  else
    docker exec "${KIND_NAME}-control-plane" ctr -n k8s.io i pull "$1" > /dev/null
  fi

  echo "Pulled image $1 in kind environemnt $KIND_NAME"
}

tag_image_k8s() {
  check_kind_version

  docker exec "${KIND_NAME}-control-plane" ctr -n k8s.io images tag --force "$1" "$2"

  echo "Tagged image $1 as $2 in kind environemnt $KIND_NAME"
}

get_latest_image_k8s() {
  check_kind_version

  local IMAGE="$1"
  docker exec "$KIND_NAME-control-plane" crictl images -o json \
    | jq -r '.images|map(select(.repoTags|any(.|startswith("'"${IMAGE%%:*}"':"))))|last|.repoTags|last'
}

load_certificate_k8s() {
  check_kind_version

  echo "Loading certificate $1 in kind environemnt $KIND_NAME..."

  "$KIND" get nodes --name "$KIND_NAME" \
    | xargs -r -n 1 -I % -P "$E2E_PARALLELISM" sh -ec "
    docker cp '$1' '%':/usr/local/share/ca-certificates/validator.crt
    docker exec '%' sh -c update-ca-certificates
    "

  echo "...done"
}

excluded_namespaces() {
  echo "calico-apiserver"
  echo "calico-system"
  echo "default"
  echo "kube-.*"
  echo "local-path-storage"
  echo "tigera-operator"
}

excluded_customresourcedefinitions() {
  echo ".*\.crd\.projectcalico\.org"
  echo ".*\.operator\.tigera\.io"
  echo ".*\.snapshot\.storage\.k8s\.io"
}

excluded_podsecuritypolicies() {
  echo "calico-.*"
  echo "tigera-operator"
  echo "csi-node-driver"
}

excluded_clusterroles() {
  echo "admin"
  echo "calico-.*"
  echo "cluster-admin"
  echo "edit"
  echo "kubeadm:.*"
  echo "local-path-provisioner-role"
  echo "system:.*"
  echo "tigera-operator"
  echo "view"
  echo "external-provisioner-runner"
  echo "external-attacher-runner"
  echo "external-snapshotter-runner"
  echo "snapshot-controller-runner"
  echo "external-resizer-runner"
  echo "external-health-monitor-controller-runner"
}

excluded_clusterrolebindings() {
  echo "calico-.*"
  echo "cluster-admin"
  echo "kubeadm:.*"
  echo "local-path-provisioner-bind"
  echo "system:.*"
  echo "tigera-operator"
  echo "csi-hostpathplugin-health-monitor-controller-cluster-role"
  echo "csi-hostpathplugin-provisioner-cluster-role"
  echo "csi-hostpathplugin-snapshotter-cluster-role"
  echo "csi-snapshotter-role"
  echo "snapshot-controller-role"
  echo "csi-hostpathplugin-resizer-cluster-role"
  echo "csi-external-health-monitor-controller-role"
  echo "csi-attacher-role"
  echo "csi-hostpathplugin-attacher-cluster-role"
  echo "csi-provisioner-role"
  echo "csi-resizer-role"
}

get_k8s_versions() {
  get_kind_images | cut -d @ -f 1 | sed 's/^v//'
}

check_kind_image_exists() {
  try_function get_kind_image "${K8S_VERSION}" > /dev/null 2>&1
  if ! "$RESULT"
  then
    echo "Kind image for k8s version ${K8S_VERSION} not found."
    echo "Available images exists only for versions: $(get_k8s_versions | tr '\n' ' ' | sed 's/ /, /g')"
    return 1
  fi
}

get_kind_image() {
  get_kind_images | sed 's/^v//' | grep "^$1[.@]"
}

get_kind_images() {
  cat << EOF
v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a
v1.33.4@sha256:25a6018e48dfcaee478f4a59af81157a437f15e6e140bf103f85a2e7cd0cbbf2
v1.32.8@sha256:abd489f042d2b644e2d033f5c2d900bc707798d075e8186cb65e3f1367a9d5a1
v1.31.12sha256:0f5cc49c5e73c0c2bb6e2df56e7df189240d83cf94edfa30946482eb08ec57d2
v1.30.13@sha256:397209b3d947d154f6641f2d0ce8d473732bd91c87d9575ade99049aa33cd648
v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec
v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251
v1.27.16@sha256:2d21a61643eafc439905e18705b8186f3296384750a835ad7a005dceb9546d20
v1.26.15@sha256:c79602a44b4056d7e48dc20f7504350f1e87530fe953428b792def00bc1076dd
v1.25.16@sha256:5da57dfc290ac3599e775e63b8b6c49c0c85d3fec771cd7d55b45fae14b38d3b
v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab
v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb3529d9fba91b3951d461edfdb
v1.22.17@sha256:f5b2e5698c6c9d6d0adc419c0deae21a425c07d81bbf3b6a6834042f25d4fba2
v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
v1.20.15@sha256:d67de8f84143adebe80a07672f370365ec7d23f93dc86866f0e29fa29ce026fe
v1.19.16@sha256:707469aac7e6805e52c3bde2a8a8050ce2b15decff60db6c5077ba9975d28b98
v1.18.20@sha256:61c9e1698c1cb19c3b1d8151a9135b379657aee23c59bde4a8d87923fcb43a91
EOF
}

create_expandable_storage_class_k8s(){
  cat << EOF | kubectl apply -f - > /dev/null
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: $KIND_EXPANDABLE_STORAGE_CLASSNAME
provisioner: rancher.io/local-path
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true
EOF

  printf '%s' "$KIND_EXPANDABLE_STORAGE_CLASSNAME"
}
